#!/usr/bin/python2
"""
This script updates the packageList ownership in Koji based on repo ownership
in Pagure.

This is heavily based on "owner-sync-pkgdb.j2" which was introduced in commit
8c2130080c by Luke Macken.
"""

# cronjobs are run on releng01.stg
# Looks like:
# /usr/local/bin/owner-sync-pkgdb f19 
# /usr/local/bin/owner-sync-pkgdb dist-5E-epel 
# /usr/local/bin/owner-sync-pkgdb dist-6E-epel
# /usr/local/bin/owner-sync-pkgdb epel7 
from __future__ import print_function
import sys
import os
import argparse
import ConfigParser
from urlparse import urljoin
import multiprocessing.pool
from math import ceil
from functools import partial

import requests
import koji

from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry


def retry_session():
    session = requests.Session()
    retry = Retry(
        total=5,
        read=5,
        connect=5,
        backoff_factor=0.3,
        status_forcelist=(500, 502, 504),
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('http://', adapter)
    session.mount('https://', adapter)
    return session


# Ansible configured global variables
STAGING = {{ 'True' if env == 'staging' else 'False' }}
HOSTNAME = '{{ inventory_hostname }}'
IPA_REALM = '{{ ipa_realm }}'
ENV_SUFFIX = '{{ env_suffix }}'
if STAGING:
    PAGURE_URL = 'https://src.stg.fedoraproject.org/'
    PDC_URL = 'https://pdc.stg.fedoraproject.org/rest_api/v1/'
else:
    PAGURE_URL = 'https://src.fedoraproject.org/'
    PDC_URL = 'https://pdc.fedoraproject.org/rest_api/v1/'
# In case the above variables end up being filled in by Ansible
if not PAGURE_URL.endswith('/'):
    PAGURE_URL = PAGURE_URL + '/'
if not PDC_URL.endswith('/'):
    PDC_URL = PDC_URL + '/'

PDC_TYPES = {
    'rpms': 'rpm',
    'modules': 'module',
    'container': 'container',
}


RAWHIDE = '28'
EXTRA_ARCH_LIST = {
    'kernel': ('i586', 'i686', 'noarch'),
    'kernel-xen-2.6': ('i586', 'i686', 'noarch'),
    'glibc': ('i686',),
    'openssl': ('i686',),
    'em8300-kmod': ('i586', 'i686'),
    'sysprof-kmod': ('i586', 'i686'),
}
VERIFY = True


def usage():
    print('Usage: owner-sync <tag>\n\t<tag>: tag to synchronize owners for',
          file=sys.stderr)
    sys.exit(1)


def get_options():
    # shamelessly stolen from koji CLI
    if STAGING:
        opts = {
            'server': 'https://koji.stg.fedoraproject.org/kojihub',
            'weburl': 'https://koji.stg.fedoraproject.org/koji',
        }
    else:
        opts = {
            'server': 'https://koji.fedoraproject.org/kojihub',
            'weburl': 'https://koji.fedoraproject.org/koji',
        }
    opts['principal'] = 'pkgdb/{0}@{1}'.format(HOSTNAME, IPA_REALM)
    opts['keytab'] = '/etc/krb5.pkgdb_{0}.keytab'.format(HOSTNAME)

    for configFile in ('/etc/koji.conf', os.path.expanduser('~/.koji/config')):
        if os.access(configFile, os.F_OK):
            f = open(configFile)
            config = ConfigParser.ConfigParser()
            config.readfp(f)
            f.close()
            if config.has_section('koji'):
                for name, value in config.items('koji'):
                    if opts.has_key(name):
                        opts[name] = value
    for entry in opts.keys():
        if entry == 'server' or entry == 'weburl':
            pass
    return opts


def get_namespace_and_version_from_tag(tag):
    if 'container' in tag:
        namespace = 'container'
        version = tag.split('-')[0].split('f')[1]
    elif 'docker' in tag:
        namespace = 'container'
        version = tag.split('-')[0].split('f')[1]
    elif 'modular' in tag:
        namespace = 'modules'
        try:
            version = tag.split('-')[0].split('f')[1]
        except IndexError:
            version = RAWHIDE
    elif tag == 'module-package-list':
        # See https://pagure.io/releng/issue/6663
        # and https://pagure.io/fm-orchestrator/issue/333
        namespace = 'rpms'
        version = RAWHIDE
    else:
        namespace = 'rpms'
        if tag.startswith('epel'):
            version = tag.split('epel')[1]
        elif tag.startswith('f'):
            version = tag.split('f')[1]
        elif tag.endswith('epel') and tag.startswith('dist'):
            # This is for older EPEL tags such as dit-6E-epel
            version = tag.split('-')[1][:-1]
        else:
            print('Error: an invalid tag was specified', file=sys.stderr)
            sys.exit(1)

    return namespace, version


def get_branch_and_arches(tag, version):
    if tag.startswith('epel'):
        # Ex: epel7 => epel7
        branch = tag
        arches = ["primary"]
    elif tag.endswith('epel'):
        # Ex: dist-6E-epel => el6
        branch = 'el%s' % version
        arches = ["primary"]
    elif tag == 'module-package-list':
        branch = 'master'
        arches = ["primary"]
    else:
        # Fedora
        if version == RAWHIDE:
            branch = 'master'
        else:
            branch = tag.split('-')[0]

        if STAGING:
            arches = ["primary"]
        else:
            if version <= "25":
                arches = ["primary", "arm", "ppc", "s390"]
            elif version <= "26":
                arches = ["primary", "s390"]
            else:
                # Yay!  Everything in primary.
                arches = ["primary"]

    return branch, arches


def get_pdc_project_name_and_branch(session, namespace, project_name,
                                    verbose=False):
    """
    Gets the branches on a project. This function is used for mapping.
    :param namespace: string of the namespace the project is in
    :param project: string of the project
    :param verbose: prints out verbose information
    :return: a tuple containing the string of the project and a list of
    branches
    """
    project_branches_url = '{0}component-branches/'.format(PDC_URL)
    params = dict(
        global_component=project_name,
        type=PDC_TYPES[namespace],
        active=True,
    )
    if verbose:
        print('- Querying {0} {1}'.format(project_branches_url, params))
    project_branches_rv = session.get(
        project_branches_url, params=params, verify=VERIFY, timeout=60)

    # If the project's branches can't be reported, just return no branches and
    # it will be skipped later on
    if not project_branches_rv.ok:
        return project_name, []

    data = project_branches_rv.json()
    return project_name, [branch['name'] for branch in data['results']]


def get_pagure_project_names_from_page(session, namespace, page,
                                       package=None, verbose=False):
    """
    Gets the names of all the Pagure projects on a page. This function is to be
    used for mapping.
    :param namespace: string of the namespace to query for projects
    :param page: int of the page to query at
    :param verbose: prints out verbose information
    :return: list of project names on the page
    """
    url = urljoin(PAGURE_URL, 'api/0/projects?namespace={0}'.format(namespace))
    url = url + '&page={0}&per_page=100&fork=false&short=true'.format(page)
    if package:
        url = url + "&pattern={0}".format(package)
    if verbose:
        print('- Querying {0}'.format(url))
    response = session.get(url, verify=VERIFY, timeout=120)
    if not bool(response):
        print("Failed to talk to %r %r." % (
            response.request.url, response), file=sys.stderr)
        return set()

    names = set()
    for project in response.json()['projects']:
        names.add(project['name'])

    return names


def get_pagure_project_branches(namespace, package=None, verbose=False):
    """
    Gets all the branches of all the Pagure projects in the desired namespace
    :param namespace: string of the namespace to query for projects
    :param verbose: prints out verbose information
    :return: dictionary in the format of {project_name: [branch_one...]}
    """
    first_page_url_path = ('api/0/projects?namespace={0}&fork=false&short=true'
                           '&page=1&per_page=1'.format(namespace))
    first_page_url = urljoin(PAGURE_URL, first_page_url_path)
    if package:
        first_page_url = first_page_url + "&pattern={0}".format(package)
    session = retry_session()
    if verbose:
        print('- Querying {0}'.format(first_page_url))
    first_page_rv = session.get(first_page_url, verify=VERIFY, timeout=120)
    if not bool(first_page_rv):
        print("Failed to talk to %r %r." % (
            first_page_rv.request.url, first_page_rv), file=sys.stderr)
        return {}
    total_projects = first_page_rv.json()['total_projects']
    num_pages = int(ceil(total_projects / 100.0))

    pool = multiprocessing.pool.ThreadPool(8)
    # Since we are going to multi-thread, we need to make a partial function
    # call so that all the function needs is an iterable to run
    partial_get_pagure_project_names_from_page = partial(
        get_pagure_project_names_from_page, session, namespace,
        package=package, verbose=verbose)
    project_names_sets = pool.map(partial_get_pagure_project_names_from_page,
                                  range(1, num_pages + 1))

    if project_names_sets:
        # Combine all the project name sets
        project_names = list(set.union(*project_names_sets))
        # Hopefully save some RAM
        del project_names_sets
    else:
        return {}

    # Since we are going to multi-thread, we need to make a partial function
    # call so that all the function needs is an iterable to run
    partial_get_pdc_project_name_and_branch = partial(
        get_pdc_project_name_and_branch, session, namespace,
        verbose=verbose)
    # Get a list of tuples in the form of (project, [branch...]), then convert
    # that to a dictionary
    project_names_to_branches = dict(pool.map(
        partial_get_pdc_project_name_and_branch, project_names))
    pool.close()
    return project_names_to_branches


def set_koji_ownership(tag, packages, arches, verbose=False):
    koji_login_options = get_options()

    koji_options={
        'krb_rdns': False,
        # About ten minutes.  The default is 12 hours.
        'timeout': 60 * 10,
    }
    for arch in arches:
        if arch == 'primary':
            session = koji.ClientSession(
                'https://koji{0}.fedoraproject.org/kojihub'.format(ENV_SUFFIX),
                opts=koji_options,
            )
        else:
            session = koji.ClientSession(
                'https://{0}.koji.fedoraproject.org/kojihub'.format(arch),
                opts=koji_options,
            )

        try:
            session.krb_login(koji_login_options['principal'], koji_login_options['keytab'])
        except:
            import traceback
            traceback.print_exc()
            print('Failed to login to "{0}" hub'.format(arch), file=sys.stderr)
            continue

        kojitag = session.getTag(tag)
        if kojitag is None:
            print('Error: tag "{0}" does not exist for arch "{1}"'
                  .format(tag, arch), file=sys.stderr)
            continue

        koji_pkgs = {}

        if verbose:
            print('- Getting a list of packages in Koji from the tag "{0}" '
                  'and arch "{1}"'.format(tag, arch))

        for p in session.listPackages(tagID=tag, inherited=True):
            koji_pkgs[p['package_name']] = p

        for pkg in packages:
            owner = 'releng'  # always
            if pkg not in koji_pkgs:
                extra_arches = None
                if pkg in EXTRA_ARCH_LIST:
                    extra_arches = EXTRA_ARCH_LIST[pkg]
                if verbose:
                    print('- Adding the package "{0}" to the package list for '
                          'the tag "{1}" on arch "{2}" and applicable extra '
                          'arches'.format(pkg, tag, arch))
                session.packageListAdd(
                    tag, pkg, owner=owner, extra_arches=extra_arches)
            elif koji_pkgs[pkg]['owner_name'] != owner:
                if verbose:
                    print('- Setting the owner on package "{0}" for the tag '
                          '"{1}" on arch "{2}"'
                          .format(pkg, tag, arch))
                session.packageListSetOwner(tag, pkg, owner, force=True)


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Process some integers.')
    parser.add_argument('tag', nargs='+',
                        help='tag to update the package list on')
    parser.add_argument('--package', help='Name of an optional single package to sync.')
    parser.add_argument('--verbose', action='store_true')
    args = parser.parse_args()
    verbose = args.verbose
    tags = args.tag
    package = args.package

    # Get all the info about the tags we are interested in
    unique_namespaces = set()
    tag_info = {}
    for tag in tags:
        namespace, version = get_namespace_and_version_from_tag(tag)
        branch, arches = get_branch_and_arches(tag, version)
        tag_info[tag] = {
            'namespace': namespace,
            'version': version,
            'branch': branch,
            'arches': arches
        }
        unique_namespaces.add(namespace)

    # Get all the project to branch mappings for every namespace
    namespace_to_projects = {}
    for namespace in unique_namespaces:
        if verbose:
            print('Querying for all the projects with the namespace "{0}"'
                  .format(namespace))
        namespace_to_projects[namespace] = \
            get_pagure_project_branches(namespace, package=package, verbose=verbose)

    for tag, info in tag_info.items():
        if verbose:
            print('Determining which projects have the namespace "{0}" and '
                  'branch "{1}"'.format(namespace, tag))
        namespace = info['namespace']
        pkgs = []
        for pkg, branches in namespace_to_projects[namespace].items():
            if info['branch'] in branches or tag == ('f' + RAWHIDE):
                # The tag and branch names are the same for "old-style" branches
                pkgs.append(pkg)
            elif namespace == 'modules':
                # Add modules to f27-modular-updates even if their only branch is '2.4'
                pkgs.append(pkg)

        # This is a special project, not in dist-git, but which needs to be in
        # the package list.
        if namespace == 'rpms':
            pkgs.append('module-build-macros')

        if verbose:
            print('Setting the Koji ownership and package list on packages in '
                  'the tag "{0}" and namespace "{1}" and for arches "{2}"'
                  .format(tag, namespace, ', '.join(info['arches'])))
        set_koji_ownership(tag, pkgs, info['arches'], verbose=verbose)

